import scrapy
from Hue.basepro import ZhengFuBaseSpider


class DalianSpider(ZhengFuBaseSpider):
    """AJAX"""
    name = 'Dalian'
    allowed_domains = ['dl.gov.cn']
    start_urls = ['http://https://www.dl.gov.cn//']
    api = "https://www.dl.gov.cn/dhc/searchJs/retrieval?question={keyword}&pageNum={page}"
    keywords = ["碳管理", "碳标识", "低碳认证技术标准", "碳排放控制制度", "低碳供应链制度", "碳达峰追踪制度", "碳合一", "碳积分", "碳评价", "碳金融", "低碳立法支持", "低碳创新机制", "低碳制度创新",]
    method = "GET"

    def edit_items_box(self, response):
        raw_data = response.json()
        try:
            items_box = raw_data['msg']['list']['全部']['list']
        except KeyError:
            self.logger.debug("KeyError from edit_items_box")
        return items_box

    def edit_page(self, response):
        raw_data = response.json()
        page = raw_data['msg']['list']['全部']['totalPage']
        return min(2000, int(page))

    def edit_items(self, items_box):
        for item in items_box:
            yield item

    def edit_item(self, item):
        data = {}
        data['title'] = item['xqtitle']
        data['url'] = item['xqurl']
        data['date'] = item['loadtime']
        return data

    def edit_article(self, response):
        article = {}
        article["source"] = ""
        article["content"] = "".join(response.css("div.art_con *::text").getall())
        return article
